#!/usr/bin/env php
<?php

# A worker that fetches and parses URLs.

$worker = new GearmanWorker();
$worker->addServer('127.0.0.1', 4730);
$worker->addFunction('fetch_url', 'fetch_url_callback');
while ($worker->work());

function fetch_url_callback($job)
{
  $url = $job->workload();

  list($protocol, $domain, $path, $file) = split_url($url);

  # Fetch document.
  $doc = new DomDocument();
  @$doc->loadHTMLFile($url);

  # Skip documents without a title.
  $nodes = $doc->getElementsByTagName('title');
  $title = '';
  foreach ($nodes as $node)
    $title .= $node->nodeValue;

  if (!$title)
    return;

  # Extract the URLs from document object.
  $xpath = new DomXpath($doc);
  $res = $xpath->query("//a/@href");
  $new_url_list = array();
  foreach($res as $href)
  {
    if (strstr($href->nodeValue, "mailto:"))
      continue;
    else if (strstr($href->nodeValue, "://"))
      $new_url = $href->nodeValue;
    else if (substr($href->nodeValue, 0, 1) == "#")
      continue;
    else if (substr($href->nodeValue, 0, 1) == "/")
      $new_url = $domain . $href->nodeValue;
    else
      $new_url = $domain . $path . $href->nodeValue;

    # Remove local references.
    if (strstr($new_url, "#"))
      $new_url = substr($new_url, 0, -strlen(strstr($new_url, "#")));

    $new_url = verify_url($new_url);
    if ($new_url)
      $new_url_list[]= $new_url;
  }

  # Remove duplicate URLs.
  $url_list = array_unique($new_url_list);

  $client = new GearmanClient();
  $client->addServer('127.0.0.1', 4730);

  # Filter URL list into local and remote URLs.
  foreach ($url_list as $new_url)
  {
    if (strstr($new_url, $domain))
    {
      print "Local URL: $new_url\n";
      $client->doBackground('insert_url', $new_url);
    }
    else
      print "Skipping Remote URL: $new_url\n";
  }

  print "Title: $title\n";

  # Send page info to the storage worker.
  $page = array(
    'url' => $url,
    'title' => $title,
    'body' => $doc->textContent
  );
  $client->doBackground('store_url', json_encode($page));
}

function split_url($url)
{
  $path = strstr(substr($url, 8), "/");
  $protocol = substr($url, 0, -(strlen(strstr($url, "://")) - 3));
  $domain = substr($url, strlen($protocol), -strlen($path));
  $file = substr(strrchr($path, "/"), 1);
  if (strlen($file) > 0)
    $path = substr($path, 0, -strlen($file));

  return array($protocol, $domain, $path, $file);
}

function verify_url($url)
{
  if (strstr($url, "://"))
  {
    if (substr($url, 0, 7) != "http://" && substr($url, 0, 8) != "https://")
      return NULL;
  }
  else
    $url= "http://$url";

  if (substr_count($url, "/") == 2)
    $url= "$url/";

  return $url;
}

?>
